In [1]:
# nltk import and downloads for tokenization, lemmetization, stemming
# Separated from inital imports for investigation upon first download
import nltk
nltk.data.path.append(r'C:\nltk_data')
nltk.download('punkt')
#tokenizer = nltk.data.load('tokenizers/punkt/PY3/english.pickle')
nltk.download('wordnet', download_dir=r'C:\nltk_data')
nltk.download('omw-1.4', download_dir=r'C:\nltk_data')
[nltk_data] Downloading package punkt to C:\nltk_data... [nltk_data] Package punkt is already up-to-date! [nltk_data] Downloading package wordnet to C:\nltk_data... [nltk_data] Package wordnet is already up-to-date! [nltk_data] Downloading package omw-1.4 to C:\nltk_data... [nltk_data] Package omw-1.4 is already up-to-date!
Out[1]:
True
In [2]:
# Imports required throughout the program
import pandas as pd
import re
import numpy as np
from sklearn.model_selection import train_test_split
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.tokenize import word_tokenize
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.svm import SVC
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, ConfusionMatrixDisplay, average_precision_score, precision_recall_curve, roc_curve, auc, roc_auc_score
import matplotlib.pyplot as plt
from sklearn.preprocessing import label_binarize
from matplotlib.ticker import PercentFormatter
from sklearn.pipeline import Pipeline
import joblib
In [3]:
# Read dataset and assign to tickets variable
tickets = pd.read_csv(r'C:\Users\abida\Documents\Uni\Year4\IndependentProject\DataSet\DataCombined.csv')
# Split the data into training and testing (80/20), stratify by 'Assignment Group' to keep balance
# https://www.geeksforgeeks.org/machine-learning/how-to-implement-stratified-sampling-with-scikit-learn/
train_data, test_data = train_test_split(
tickets,
test_size=0.2,
stratify=tickets['Assignment Group'])
# Display top rows of each data set to show separation
display(train_data.head())
display(test_data.head())
| Title | Description | Assignment Group | |
|---|---|---|---|
| 1919 | Unable to Complete V5C Update Due to Session E... | Customer modifying address for vehicle ZM88 NO... | Vehicle Department |
| 60 | Customer unable to change address on driving l... | Driver Sarah Jenkins (DLN: J12345678) reports ... | Drivers Department |
| 894 | Incorrect Price Displayed for Private mark | Customer attempted to purchase private registr... | Private Registration Department |
| 127 | Licence details don't match online | Endorsement for using mobile phone (CU80) expi... | Drivers Department |
| 491 | Urgent: Firewall Restricting Web Access | A firewall is actively restricting access to a... | Network Department |
| Title | Description | Assignment Group | |
|---|---|---|---|
| 577 | High Severity Vulnerability in test environment | High severity vulnerabilities have been detect... | Network Department |
| 66 | Driving licence address change - address not f... | Customer Emily White (DOB: 03/09/1992 Licence ... | Drivers Department |
| 1177 | Transaction error on private registration purc... | Customer reported error while purchasing 'ZX90... | Private Registration Department |
| 657 | Create New Firewall Policy | Firewall policy needed to permit traffic on po... | Network Department |
| 1710 | No Delivery of Replacement V5C | Customer reports that the V5C replacement for ... | Vehicle Department |
In [4]:
""" Pre-processing steps: change all text to lowercase, removes characters, all whitespace,
tokenization, stemming and lemmatization """
#https://www.geeksforgeeks.org/machine-learning/python-stemming-words-with-nltk/
#https://www.geeksforgeeks.org/python/python-lemmatization-with-nltk/
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()
# Create clean_data function
def clean_data(data):
data = str(data).lower()
data = re.sub(r'\W+', ' ', data)
data = re.sub(r'\s+', ' ', data).strip()
tokens = word_tokenize(data)
stemmed_tokens = [stemmer.stem(token) for token in tokens]
lemmatized_tokens = [lemmatizer.lemmatize(token) for token in stemmed_tokens]
return ' '.join(lemmatized_tokens)
In [5]:
# Loops through each field, applying clean_data function
# Combines title and description to one column for classification
for data in [train_data, test_data]:
data['Title'] = data['Title'].apply(clean_data)
data['Description'] = data['Description'].apply(clean_data)
data['Combined_Text'] = data['Title'] + ' ' + data['Description']
# Print top few lines of cleaned data to check function has actioned as expected
display(train_data[['Title', 'Description', 'Combined_Text']].head())
| Title | Description | Combined_Text | |
|---|---|---|---|
| 1919 | unabl to complet v5c updat due to session expiri | custom modifi address for vehicl zm88 nop expe... | unabl to complet v5c updat due to session expi... |
| 60 | custom unabl to chang address on drive licenc | driver sarah jenkin dln j12345678 report her n... | custom unabl to chang address on drive licenc ... |
| 894 | incorrect price display for privat mark | custom attempt to purchas privat registr mark ... | incorrect price display for privat mark custom... |
| 127 | licenc detail don t match onlin | endors for use mobil phone cu80 expir in march... | licenc detail don t match onlin endors for use... |
| 491 | urgent firewal restrict web access | a firewal is activ restrict access to a websit... | urgent firewal restrict web access a firewal i... |
In [6]:
# Assigns Vectorizer as variable for TF-IDF technique where stop words are excluded and the first 1000 words are used.
vectorizer = TfidfVectorizer(max_features=1000, stop_words='english')
# Applies Vectorizer to identify terms in training set and applies to testing set.
#https://www.geeksforgeeks.org/nlp/how-to-store-a-tfidfvectorizer-for-future-use-in-scikit-learn/
x_train = vectorizer.fit_transform(train_data['Combined_Text'])
x_test = vectorizer.transform(test_data['Combined_Text'])
# Identifies Assignment Group as the 'target'.
y_train = train_data['Assignment Group']
y_test = test_data['Assignment Group']
In [7]:
# Advised by Anaconda Assistant
# Check for duplicates in data set
train_texts = set(train_data['Combined_Text'])
test_texts = set(test_data['Combined_Text'])
overlap = train_texts.intersection(test_texts)
print(f"Number of identical texts in both sets: {len(overlap)}")
print(f"Percentage of test set: {len(overlap)/len(test_texts)*100:.2f}%")
# Show some examples
duplicates = tickets[tickets.duplicated(subset=['Description'], keep='first')]
print("\nExample duplicates:")
print(duplicates[['Description']])
Number of identical texts in both sets: 0 Percentage of test set: 0.00% Example duplicates: Empty DataFrame Columns: [Description] Index: []
In [8]:
# Creates linear SVM (high-dimensional data)
# Balanced data set = 5 classes, with 40 tickets of each type (8 types)
SVM = SVC(kernel='linear', class_weight='balanced', C=1, probability=True)
SVM.fit(x_train, y_train)
# SVM learns to predict
y_predict = SVM.predict(x_test)
# Evaluation Report of the model performance
# https://stackoverflow.com/questions/22022410/scikit-classification-report-change-the-format-of-displayed-results
print("Classification Report:")
print(classification_report(y_test, y_predict, digits=4))
# Included for testing purposing, e.g. check for overfitting
# https://www.geeksforgeeks.org/machine-learning/how-to-avoid-overfitting-in-svm/
train_accuracy = accuracy_score(y_train, SVM.predict(x_train))
test_accuracy = accuracy_score(y_test, y_predict)
print('\nTraining Accuracy:\n', train_accuracy)
print('\nTesting Accuracy:\n', test_accuracy)
Classification Report:
precision recall f1-score support
Drivers Department 1.0000 0.9875 0.9937 80
Network Department 1.0000 1.0000 1.0000 80
Private Registration Department 1.0000 1.0000 1.0000 80
Service Desk Department 1.0000 1.0000 1.0000 80
Vehicle Department 0.9877 1.0000 0.9938 80
accuracy 0.9975 400
macro avg 0.9975 0.9975 0.9975 400
weighted avg 0.9975 0.9975 0.9975 400
Training Accuracy:
1.0
Testing Accuracy:
0.9975
In [9]:
# Create confusion matrix to show results
# https://stackoverflow.com/questions/76775348/multiclass-confusion-matrix-in-python
cm = confusion_matrix(y_test, y_predict, labels=SVM.classes_)
disp = ConfusionMatrixDisplay(cm, display_labels=SVM.classes_)
disp.plot(cmap="Blues")
plt.title("Confusion Matrix")
plt.xlabel("Predicted")
plt.ylabel("Actual")
plt.xticks(rotation=90)
plt.show()
In [10]:
# Assisted by Anaconda Assistant
# Set SVM class names to variable
classes = SVM.classes_
# Binarise the output for multi-class as PR curve is not compatible with multi-class
y_test_bin = label_binarize(y_test, classes=classes)
n_classes = y_test_bin.shape[1]
# Calculate probability scores
y_score = SVM.predict_proba(x_test)
# Calculations for Precision-Recall curve and Average Precision for each class
precision = dict()
recall = dict()
average_precision = dict()
# for loop to action calculations of PR curve and average precision for each class
for i in range(n_classes):
precision[i], recall[i], _ = precision_recall_curve(y_test_bin[:, i], y_score[:, i])
average_precision[i] = average_precision_score(y_test_bin[:, i], y_score[:, i])
# Calculate micro-average of precision average using sklearn's built-in functionality
average_precision["micro"] = average_precision_score(y_test_bin, y_score, average="micro")
# Create pairs of binary indicators and scores
all_y_true = []
all_y_score = []
for i in range(n_classes):
all_y_true.extend(y_test_bin[:, i])
all_y_score.extend(y_score[:, i])
precision["micro"], recall["micro"], _ = precision_recall_curve(all_y_true, all_y_score)
# Calculate macro-average AP (Average Precision)
average_precision["macro"] = np.mean([average_precision[i] for i in range(n_classes)])
# Plot the PR curve
plt.figure(figsize=(10, 8))
# Plot micro-average PR curve
plt.plot(recall["micro"], precision["micro"],
label=f'Micro-average (AP = {average_precision["micro"]:.2f})',
color='deeppink', linestyle='-', linewidth=2)
# Add the graph details
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curves')
plt.legend(loc="best")
plt.grid(True)
# Calculate the baseline for random classifier
n_pos = np.sum(y_test_bin, axis=0)
baseline = n_pos / len(y_test_bin)
plt.plot([0, 1], [baseline.mean(), baseline.mean()],
linestyle='--', color='gray', label='Random')
plt.show()
In [11]:
# Assisted by Anaconda Assistant
# Example class list and binarization
classes = np.unique(y_test)
n_classes = len(classes)
y_test_bin = label_binarize(y_test, classes=classes)
y_score = SVM.predict_proba(x_test)
# Calculate ROC curve and ROC area for each class
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(n_classes):
fpr[i], tpr[i], _ = roc_curve(y_test_bin[:, i], y_score[:, i])
roc_auc[i] = auc(fpr[i], tpr[i])
# Calculate micro-average ROC curve and AUC
fpr["micro"], tpr["micro"], _ = roc_curve(y_test_bin.ravel(), y_score.ravel())
roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
# Set up graphs (ax1 = ROC, ax2 = AUC-ROC)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6))
# Plot ROC curve for each class on ax1 (ROC Curve)
for i in range(n_classes):
ax1.plot(fpr[i], tpr[i], lw=2, label=f'Class {classes[i]} (AUC = {roc_auc[i]:.2f})')
ax1.plot([0, 1], [0, 1], 'k--', lw=2)
ax1.set_xlim([0.0, 1.0])
ax1.set_ylim([0.0, 1.05])
ax1.set_xlabel('False Positive')
ax1.set_ylabel('True Positive')
ax1.set_title('Multiclass ROC Curve Graph')
ax1.legend(loc="lower right")
ax1.grid(True, alpha=0.3)
ax1.xaxis.set_major_formatter(PercentFormatter(1.0))
ax1.yaxis.set_major_formatter(PercentFormatter(1.0))
# Plot micro-average ROC curve with AUC filled on ax2 (AUC-ROC)
ax2.plot(fpr["micro"], tpr["micro"], color='darkorange', lw=2, label=f'Micro-average ROC (AUC = {roc_auc["micro"]:.2f})')
ax2.fill_between(fpr["micro"], tpr["micro"], 0, color='darkorange', alpha=0.2)
ax2.plot([0, 1], [0, 1], 'k--', lw=2)
ax2.set_xlim([0.0, 1.0])
ax2.set_ylim([0.0, 1.05])
ax2.set_xlabel('False Positive')
ax2.set_ylabel('True Positive')
ax2.set_title('AUC-ROC Curve Graph\nAUC Highlighted (average ROC)')
ax2.legend(loc="lower right")
ax2.grid(True, alpha=0.3)
ax2.xaxis.set_major_formatter(PercentFormatter(1.0))
ax2.yaxis.set_major_formatter(PercentFormatter(1.0))
# Display plots
plt.tight_layout()
plt.show()
In [12]:
# Anaconda Assistant
pipeline = Pipeline([('vectorizer', vectorizer),('svm', SVM)])
# Save the pipeline to file for use with API
joblib.dump(pipeline, "svm_pipeline.pkl")
Out[12]:
['svm_pipeline.pkl']